import scrapy
from ..items import MyspiderItem


class DemoSpider(scrapy.Spider):
    name = 'demo'
    # allowed_domains = ['www.baidu.com']
    #https://quotes.toscrape.com/page/2/
    format_url = 'https://quotes.toscrape.com/page/{}/'
    page = 1
    next_url = format_url.format(page)
    start_urls = ['https://quotes.toscrape.com/']

    def parse(self, response):
        divs = response.xpath('//div[@class="quote"]')
        print(len(divs))
        for div in divs:
            item = MyspiderItem()
            item['text'] = div.xpath('./span[1]/text()').get()
            item['author'] = div.xpath('./span[2]/small/text()').get()
            item['tags'] = div.xpath('./div//a/text()').getall()
            print(item)
            yield item

        if self.page < 5:
            self.page = self.page + 1
            print(f"第{self.page}页已经爬完")
            yield scrapy.Request(url=self.format_url.format(self.page), callback=self.parse)

    # def start_requests(self):
    #     for page in (1, 5):
    #         format_url = 'https://quotes.toscrape.com/page/{}/'
    #         yield scrapy.Request(url=format_url.format(page), callback=self.parse)

